# *-* coding:utf8 *-*

# 需要安装,发送请求用
import urllib.parse
import json
import requests
# 需要安装,解析数据用
from lxml import etree
from selenium import webdriver
# 正则
import re

from spider_server.conf.config_util import ConfigUtil
from spider_server.email_smtp.email_util import EmailSMTP
from spider_server.exam_spider_gsrst import exam_db
from spider_server.logs.logger import Logger
from spider_server.message_sms.message_util import TencentSMS
from spider_server.scheduler.scheduler_db import SchedulerDB
from time import sleep

"""
1 准备URL列表
2 遍历URL,发送请求,获取响应数据
3 解析数据
4 保存数据
"""
logger = Logger(__name__).get_log()


class ExamServer(object):
    def __init__(self):
        """初始化数据"""
        # 准备url模板
        self.url = ConfigUtil().get("URL", "URL_GSRST_ZYJS")
        # 指定请求头
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36",
            "Cookie": "TNU5z5PTPMUxS = 5hjbSmm3kNkYOeR2ZQun1Ne92Z4WY4Q2iShErt3ou_oJzC7I4H4yAVCckL9XopZRCPQ2j9YAeUOWeHpE00wyenA;TNU5z5PTPMUxT = 53fT6KDEw8m9qqqmdASLSsapAMGxrhWrEbWjyXCQWUs6riEbk2JXn9fs.DDfaLQXA03pUt6hndg7_BySS1caJR4N5HL3mkTBYLs8BIFV4viHL_P06_ykeOKwQDD6v.EfCFWMtmiPGIC2dlkFES4AstqltEBoRoq7JalxcqOoiECdyAQ6ml_YbVehfzz9EmIlphZirKBsP1533bMUKGHQQZbohbyaYAkZoR7HL.TRU6IymM5w2n_yiuRwdDtQvHUHLxP4RKVToK0gHTpVvorgxxfxCTfgWsy_t692ZfW5Mlk5G"
        }

    def run(self):
        """程序入口,核心入口"""
        # 1 发送请求,获取响应数据
        page = self.get_page_from_url(self.url)
        # 2 解析数据
        datas = self.get_datas_from_page(page)
        # 3 保存数据
        self.save_data(datas)

    def get_page_from_url(self, url):
        """根据url,发送请求,获取接口数据"""
        # 网站做了反爬虫限制,此处用了chromedriver.exe,需要下载,并放置python根目录

        chrome_options = webdriver.ChromeOptions()
        # 无头模式,启用后拿不到数据(网站做了什么限制),不启用linux环境报错,只能在windows环境搞了
        # chrome_options.add_argument('--headless')
        chrome_options.add_argument('--no-sandbox')
        chrome_options.add_argument('--disable-gpu')
        chrome_options.add_argument('--disable-dev-shm-usage')

        # 正式环境驱动路径linux
        # driver = webdriver.Chrome('/usr/bin/chromedriver', chrome_options=chrome_options)

        # 测试环境windows,驱动放在了D:\Python37\Scripts目录下,路径可以免写
        driver = webdriver.Chrome(chrome_options=chrome_options)

        # 被网站检测出来是selenium不让爬了,解决方案如下
        driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
            "source":
            """
                Object.defineProperty(navigator, 'webdriver', {
                  get: () => undefined
                })
            """
        })
        driver.get(url)
        sleep(3)
        # 获取源码，解析
        html = driver.page_source
        return html

    def get_datas_from_page(self, page):
        """解析页面数据"""
        # 页面转换为Element,就可以使用Xpath提取数据了
        element = etree.HTML(page)
        # 获取标签列表
        # xpath返回的是一个列表
        lis = element.xpath("//*[@id='component_1758']/div/ul/li")
        # 遍历列表,提取需要的数据
        data_list = []
        logger.info(len(lis))
        for li in lis:
            item = {}
            # ./ 表示当前路径之下,// 表示获取该节点及其之下所有的文本
            # 通知名称
            item["notice_name"] = li.xpath("./a/span//text()")[0]
            # 通知时间
            item["notice_time"] = li.xpath("./span[2]//text()")[0]
            # 通知url
            item["notice_url"] = urllib.parse.urljoin(self.url,
                                                  li.xpath("./a/@href")[0])
            data_list.append(item)
        return data_list

    def save_data(self, datas):
        """保存数据"""
        for data in datas:
            # 查看数据是否已经入库
            count = exam_db.ExamDB().get_exam_count(data["notice_name"])
            if count == 0:
                # 新通知处理
                logger.info("考试通知--->%s" % data["notice_name"])
                # 邮件通知
                job_detail = SchedulerDB().get_job_detail("exam_job")
                if job_detail["email_notice"] == "1":
                    EmailSMTP().send_email("甘肃人社厅-人事考试", "标题：%s \n 时间：%s \n 链接：%s" % (
                        data["notice_name"], data["notice_time"], data["notice_url"]))
                # 短信通知
                if job_detail["message_notice"] == "1":
                    TencentSMS().send_message("甘肃人社厅-人事考试")
                # 数据入库
                exam_db.ExamDB().save_exam_data(data)

    def get_exam_lists(self, name):
        """返回考试信息列表"""
        results = exam_db.ExamDB().get_exam_lists(name)
        return results


if __name__ == '__main__':
    ExamServer().run()
